『PyTorch × TensorFlow』第十七弹_ResNet快速实现
对比之前的复杂版本，这次的torch实现其实简单了不少，不过这和上面的代码实现逻辑过于复杂也有关系。
一、PyTorch实现
# Author : hellcat
# Time : 18-3-2
"""
import os
os.environ["CUDA_VISIBLE_DEVICES"]="-1"
import numpy as np
np.set_printoptions(threshold=np.inf)
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
"""
import torch as t
import torch.nn as nn
from torch.nn import functional as F
class ResidualBlock(nn.Module):
def __init__(self, inchannel, outchannel, stride=1, shortcut=None):
super(ResidualBlock, self).__init__()
self.left = nn.Sequential(
nn.Conv2d(inchannel, outchannel, kernel_size=3, stride=stride, padding=1),
nn.BatchNorm2d(outchannel),
nn.ReLU(inplace=True),
nn.Conv2d(outchannel, outchannel, 3, 1, 1, bias=False),
nn.BatchNorm2d(outchannel)
)
self.right = shortcut
def forward(self, x):
out = self.left(x)
residual = x if self.right is None else self.right(x)
out += residual
return F.relu(out)
class ResNet(nn.Module):
def __init__(self, num_classes=1000):
super(ResNet, self).__init__()
self.pre = nn.Sequential(
nn.Conv2d(3, 64, 7, 2, 3, bias=False),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
)
self.layer1 = self._make_layer(inchannel=64, outchannel=128, block_num=3)
self.layer2 = self._make_layer(inchannel=128, outchannel=256, block_num=4, stride=2)
self.layer3 = self._make_layer(inchannel=256, outchannel=512, block_num=6, stride=2)
self.layer4 = self._make_layer(inchannel=512, outchannel=512, block_num=3, stride=2)
self.fc = nn.Linear(512, num_classes)
def _make_layer(self, inchannel, outchannel, block_num, stride=1):
shortcut = nn.Sequential(
nn.Conv2d(inchannel, outchannel, 1, stride, bias=False),
nn.BatchNorm2d(outchannel)
)
layers = []
layers.append(ResidualBlock(inchannel, outchannel, stride, shortcut))
for i in range(1, block_num):
layers.append(ResidualBlock(outchannel, outchannel))
return nn.Sequential(*layers)
def forward(self, x):
x = self.pre(x) # [1, 64, 56, 56]
x = self.layer1(x) # [1, 128, 56, 56]
x = self.layer2(x) # [1, 256, 28, 28]
x = self.layer3(x) # [1, 512, 14, 14]
x = self.layer4(x) # [1, 512, 7, 7]
x = F.avg_pool2d(x, 7)
x = x.view(x.size(0), -1)
return self.fc(x)
def hook(module, inputdata, output):
'''把这层的输出拷贝到features中'''
print("钩子输出：", output.data.size())
module = ResNet()
img = t.autograd.Variable(t.randn(1, 3, 224, 224))
handle = module.pre[0].register_forward_hook(hook)
out = module(img)
handle.remove()
print(out)
上面代码中，我们注册了钩子尝试分析一下中间的输出，可以看到，torch中的卷积层默认是SAME模式，输出就是in/stride，和TensorFlow一致，
torch.Size([1, 64, 112, 112])
Variable containing:
0.6336 -0.5863 0.6472 ... -0.4694 0.1808 0.2837
[torch.FloatTensor of size 1x1000]
二、TensorFlow实现
同样的逻辑下，ResNet34的TensorFlow实现如下，使用的封装包ops，之前有介绍过，这里面小修了卷积层的封装，使得conv2d可以舍弃bias(就是卷及计算后不加偏执)，
# Author : hellcat
# Time : 18-3-7
"""
import os
os.environ["CUDA_VISIBLE_DEVICES"]="-1"
import numpy as np
np.set_printoptions(threshold=np.inf)
"""
import ops
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
def ResidualBlock(x,
outchannel, stride=1, shortcut=None,
train=True, name="ResidualBlock"):
with tf.variable_scope(name):
conv1 = ops.conv2d(x, outchannel,
k_h=3, k_w=3,
s_h=stride, s_w=stride, scope="conv1")
bn1 = tf.nn.relu(ops.batch_normal(conv1, train=train, scope="bn1"))
conv2 = ops.conv2d(bn1, outchannel,
k_h=3, k_w=3,
s_h=1, s_w=1,
with_bias=False, scope="conv2")
left = ops.batch_normal(conv2, train=train, scope="bn2")
right = x if shortcut is None else shortcut(x)
return tf.nn.relu(left + right)
class ResNet():
def __init__(self):
with tf.variable_scope("input"):
x = tf.placeholder(dtype=tf.float32, shape=[1, 224, 224, 3])
with tf.variable_scope("pre"):
conv = ops.conv2d(x, output_dim=64,
k_h=7, k_w=7,
s_h=2, s_w=2,
with_bias=False)
bn = tf.nn.relu(ops.batch_normal(conv))
pool = tf.nn.max_pool(bn, ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope("layer1"):
layer1 = self._make_layer(pool, outchannel=128, block_num=3)
with tf.variable_scope("layer2"):
layer2 = self._make_layer(layer1, outchannel=256, block_num=4, stride=2)
with tf.variable_scope("layer3"):
layer3 = self._make_layer(layer2, outchannel=512, block_num=6, stride=2)
with tf.variable_scope("layer4"):
layer4 = self._make_layer(layer3, outchannel=512, block_num=3, stride=2)
# Tensor("layer1/ResidualBlock2/Relu_1:0", shape=(1, 56, 56, 128), dtype=float32)
# Tensor("layer2/ResidualBlock3/Relu_1:0", shape=(1, 28, 28, 256), dtype=float32)
# Tensor("layer3/ResidualBlock5/Relu_1:0", shape=(1, 14, 14, 512), dtype=float32)
# Tensor("layer4/ResidualBlock2/Relu_1:0", shape=(1, 7, 7, 512), dtype=float32)
pool = tf.nn.avg_pool(layer4, ksize=[1, 7, 7, 1],
strides=[1, 7, 7, 1], padding='SAME')
reshape = tf.reshape(pool, [layer4.get_shape()[0], -1])
self.fc = ops.linear(reshape, 1000)
def __call__(self, *args, **kwargs):
return self.fc
def _make_layer(self,x,
outchannel,
block_num, stride=1):
def shortcut(input_):
with tf.variable_scope("shortcut"):
conv = ops.conv2d(input_, output_dim=outchannel,
k_w=1, k_h=1, s_w=stride, s_h=stride,
with_bias=False)
return ops.batch_normal(conv)
x = ResidualBlock(x, outchannel, stride,
shortcut, name="ResidualBlock0")
for i in range(1, block_num):
x = ResidualBlock(x, outchannel,
name="ResidualBlock{}".format(i))
return x
if __name__ == "__main__":
resnet = ResNet()
print(resnet())
ops.py卷积修改封装如下,
def conv2d(input_, output_dim,
k_h=5, k_w=5, s_h=2, s_w=2, stddev=0.02,
scope="conv2d", with_w=False, with_bias=True):
"""
卷积网络封装
:param input_:
:param output_dim: 输出的feature数目
:param k_h:
:param k_w:
:param s_h:
:param s_w:
:param stddev:
:param scope:
:param with_w:
:param with_bias: 是否含有bias层
:return:
"""
with tf.variable_scope(scope):
w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim],
initializer=tf.truncated_normal_initializer(stddev=stddev))
conv = tf.nn.conv2d(input_, w, strides=[1, s_h, s_w, 1], padding='SAME')
if with_bias:
biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
else:
biases = None
if with_w:
return conv, w, biases
else:
return conv
输出如下，
Tensor("Linear/add:0", shape=(1, 1000), dtype=float32)
附：
ops.py截止本文发布的最新版本状态，
# Author : hellcat
# Time : 18-1-21
# Usage : 网络层函数封装
"""
conv2d
deconv2d
lrelu
linear
"""
import tensorflow as tf
# def batch_normal(x, train=True, epsilon=1e-5, decay=0.9, scope="batch_norm"):
# return tf.contrib.layers.batch_norm(x,
# decay=decay,
# updates_collections=None,
# epsilon=epsilon,
# scale=True,
# is_training=train,
# scope=scope)
def batch_normal(x, epsilon=1e-5, momentum=0.9, train=True, scope='batch_norm'):
with tf.variable_scope(scope):
return tf.contrib.layers.batch_norm(x,
decay=momentum,
updates_collections=None,
epsilon=epsilon,
scale=True,
is_training=train)
'''
Note: when training, the moving_mean and moving_variance need to be updated.
By default the update ops are placed in `tf.GraphKeys.UPDATE_OPS`, so they
need to be added as a dependency to the `train_op`. For example:
```python
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss)
```
One can set updates_collections=None to force the updates in place, but that
can have a speed penalty, especially in distributed settings.
'''
# class batch_norm(object):
# def __init__(self, epsilon=1e-5, decay=0.9, scope="batch_norm"):
# with tf.variable_scope(scope):
# self.epsilon = epsilon
# self.decay = decay
# # self.scope = scope
#
# def __call__(self, x, scope, train=True):
# return tf.contrib.layers.batch_norm(x,
# decay=self.decay,
# updates_collections=None,
# epsilon=self.epsilon,
# scale=True,
# is_training=train,
# scope=scope)
def concat(tensor_a, tensor_b):
"""
组合Tensor,注意的是这里tensor_a的宽高应该大于等于tensor_b
:param tensor_a: 前面的tensor
:param tensor_b: 后面的tensor
:return:
"""
if tensor_a.get_shape().as_list()[1] > tensor_b.get_shape().as_list()[1]:
return tf.concat([tf.slice(tensor_a,
begin=[0, (int(tensor_a.shape[1]) - int(tensor_b.shape[1])) // 2,
(int(tensor_a.shape[1]) - int(tensor_b.shape[1])) // 2, 0],
size=[int(tensor_b.shape[0]), int(tensor_b.shape[1]),
int(tensor_b.shape[2]), int(tensor_a.shape[3])],
name='slice'),
tensor_b],
axis=3, name='concat')
elif tensor_a.get_shape().as_list()[1] < tensor_b.get_shape().as_list()[1]:
return tf.concat([tensor_a,
tf.slice(tensor_b,
begin=[0, (int(tensor_b.shape[1]) - int(tensor_a.shape[1])) // 2,
(int(tensor_b.shape[1]) - int(tensor_a.shape[1])) // 2, 0],
size=[int(tensor_a.shape[0]), int(tensor_a.shape[1]),
int(tensor_a.shape[2]), int(tensor_b.shape[3])],
name='slice')],
axis=3, name='concat')
else:
return tf.concat([tensor_a, tensor_b], axis=3)
def conv_cond_concat(x, y):
"""
广播并连接向量,用于ac_gan的标签对矩阵拼接
:param x: features，例如shape：[n,16,16,128]
:param y: 扩暂维度后的标签，例如shape：[n,1,1,10]
:return: 拼接后features，例如：[n,16,16,138]
"""
x_shapes = x.get_shape()
y_shapes = y.get_shape()
return tf.concat([x, y * tf.ones([x_shapes[0], x_shapes[1], x_shapes[2], y_shapes[3]])], axis=3)
def conv2d(input_, output_dim,
k_h=5, k_w=5, s_h=2, s_w=2, stddev=0.02,
scope="conv2d", with_w=False, with_bias=True):
"""
卷积网络封装
:param input_:
:param output_dim: 输出的feature数目
:param k_h:
:param k_w:
:param s_h:
:param s_w:
:param stddev:
:param scope:
:param with_w:
:param with_bias: 是否含有bias层
:return:
"""
with tf.variable_scope(scope):
w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim],
initializer=tf.truncated_normal_initializer(stddev=stddev))
conv = tf.nn.conv2d(input_, w, strides=[1, s_h, s_w, 1], padding='SAME')
if with_bias:
biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape())
else:
biases = None
if with_w:
return conv, w, biases
else:
return conv
def deconv2d(input_, output_shape,
k_h=5, k_w=5, s_h=2, s_w=2, stddev=0.02,
scope="deconv2d", with_w=False):
"""
转置卷积网络封装
:param input_:
:param output_shape: 输出的shape
:param k_h:
:param k_w:
:param s_h:
:param s_w:
:param stddev:
:param scope:
:param with_w:
:return:
"""
with tf.variable_scope(scope):
# filter : [height, width, output_channels, in_channels]
w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
initializer=tf.random_normal_initializer(stddev=stddev))
try:
deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape,
strides=[1, s_h, s_w, 1])
# Support for verisons of TensorFlow before 0.7.0
except AttributeError:
deconv = tf.nn.deconv2d(input_, w, output_shape=output_shape,
strides=[1, s_h, s_w, 1])
biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0))
deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())
if with_w:
return deconv, w, biases
else:
return deconv
def lrelu(x, leak=0.2):
"""
Leak_Relu层封装
:param x:
:param leak:
:return:
"""
return tf.maximum(x, leak*x)
def linear(input_, output_size,
stddev=0.02, bias_start=0.0,
scope=None, with_w=False):
"""
全连接层封装
:param input_:
:param output_size: 输出节点数目
:param scope:
:param stddev:
:param bias_start: 使用常数初始化偏执，常数值设定
:param with_w: 返回是否返回参数Variable
:return:
"""
shape = input_.get_shape().as_list()
with tf.variable_scope(scope or "Linear"):
matrix = tf.get_variable("Matrix", [shape[1], output_size], tf.float32,
tf.random_normal_initializer(stddev=stddev))
bias = tf.get_variable("bias", [output_size], initializer=tf.constant_initializer(bias_start))
if with_w:
return tf.matmul(input_, matrix) + bias, matrix, bias
else:
return tf.matmul(input_, matrix) + bias
